{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### softmax函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$y_k = \\frac{exp(a_k)}{\\sum_{i=1}^{n} exp(a_i)}$$"
   ]
  },
  {
   "attachments": {
    "a.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKEAAADXCAYAAAByKjFqAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO2dd1hT5/v/75MBAUIIIEMQscp0FK1QsVUREK0Dq1ZFRNRapbi1Wlst1lXH1w92KHVg66gbZ8VtXTixrQMVkeIAB4IoIxB23r8//CUVSUJCTghYX9eV6wLOc57nDuc+z7wHAwD0lrcYjoccQ0vwlre8VcK3GJy3SvgWg8MztABvqXsqKyvpwYMH9PTpU5JKpZSXl0fm5uZkampKNjY21KJFCzIyMqozed4q4X+A58+f08mTJ+nUqVN07tw5Sk1NpdLSUpXleTweNWvWjDp27EgBAQEUGBhITk5OepOPebs6fjMpLy+nffv20W+//UZHjx4lLpdLvr6+1LlzZ/L09CQ3Nzeyt7cnU1NTsrS0pIKCApJKpfTs2TNKTU2l1NRUunDhAiUkJJBEIqEPP/yQwsPDKTQ0lMzNzdkU9aFelVAmk9H169fp7NmzdPv2bbpz5w49ffqUioqKSCKRkEgkIjMzM3JwcCA3Nzdq3bo1denShVq2bEkMw+hLrDeakpIS+vXXXyk6OpoeP35MH330EYWHh1OfPn3IxMRE6/oqKiro9OnTtGnTJtqzZw8ZGRnRxIkTafLkyWRpacmGyOwrYWVlJR09epS2bNlCR44coRcvXlDjxo2rvH1mZmYkEokoPz+fioqKKDMzk+7cuUO3bt2inJwcsrW1pd69e9Pw4cOpS5cuxOG8XT9pwpEjR2jixIn0+PFjGjVqFM2YMYOaNm3KWv25ubm0YsUKWr58OTEMQ4sXL6bPPvtM1w7jIYElCgsLsWzZMjRp0gQcDgddunTB6tWrkZqaqlU9t27dwk8//YSOHTuCiNC8eXOsXr0aJSUlbIn6xiGRSBAeHg4iwqBBg/Dw4UO9tpefn4/JkyeDx+PBz88Pjx490qW6DJ2VsLKyErGxsWjUqBGEQiG++OILpKen61otAOD27dsYO3YsjI2N0aRJE+zYsYOVet8kkpKS4ObmBjs7Oxw8eLBO275y5Qo8PT1hY2ODI0eO1LYa3ZQwNTUVH3zwAXg8HqZMmYKcnBxdqlPJkydP8Nlnn4FhGHTv3l3XN++N4ezZsxCLxfD390dmZqZBZCgqKsLIkSPB5/Px22+/1aaK2ithXFwcRCIRvL29cf369dpWoxXnz5+Hh4cHbGxscOzYsTpps75y4sQJmJiYICwsDGVlZYYWBwsWLADDMFi9erW2t9ZOCZcuXQoOh4MJEybU+VxNIpEgLCwMXC4Xv/zyS522XV+4dOkSzM3NMX78eMhkMkOLo2D16tXgcrnYvHmzNrdpr4RRUVH1QgGWLFkCDoeD6Ohog8pR1zx+/Bg2NjYICwurVwooZ/HixTAyMsKlS5c0vUU7JZQ3sGfPHu2l0wO//PILOBwONm3aZGhR6oSKigp06dIFfn5+KC0tNbQ4KhkzZgycnZ3x4sULTYprroS7d+8Gh8PRtqvVOytWrACXy8Xp06cNLYreWbRoEezs7PDkyRNDi6KWkpIStGvXDqGhoZoU10wJ7927BwsLCyxcuFA36fTEF198AQcHB2RlZRlaFL1x//59mJqa1vk2TG25c+cOBAIBTp48WVPRjBpPTABQly5dyMLCguLj4+vlcVpZWRl17dqV7O3tac+ePYYWRy/079+fuFwu7dq1y9CiaMzcuXNpx44dlJSURHw+X1Wxmk9M1q9fDxsbGzx79oyF90N/NLSeQhsSExPB4/GQlpZmaFG0QiqVwtbWFr/++qu6YuqH47y8PNjY2NRUSb1h8eLFcHV1rdeT9trQt29fhIWFGVqMWrFkyRK4urqivLxcVRH1Srhw4UK4u7ujsrKSfen0gFQqhb29PWJjYw0tCmv8888/YBgGf/75p6FFqRV5eXkwNTXF3r17VRVRrYRFRUVo1KgR1q9frxfh9MXSpUvRokULdW9eg+Lbb79Fq1atDC2GTgwZMgT9+vVTdVm1Eq5fvx5WVlYNznrlxYsXMDY2Rnx8vKFFYYUWLVpg0aJFhhZDJ+Lj42FkZITc3FxllzNUGupt3LiRBg8eTMbGxvpZOukJS0tL6tOnD23atMnQoujM3bt36e7du9SjRw+9t5Wfn0+XLl2itWvXUt++fVmt29/fn4iITp8+rbyAMtXMysoCh8PRZI+nXrJ9+3aYmJiguLjY0KLoRGxsLCwtLetkTh4VFYWoqCgQEVSohU507twZEyZMUHZJeU946tQpMjY2po4dO7L6RryOvt6+wMBAKikpoQsXLrBWpy5cuXKF9u3bR8XFxVrdl5iYSD4+PnViWb5gwQJasGCB3urv0KEDXb58Wek1pd/uzJkz5OvrSwKBQG9CERFFR0fTwYMHKSIiguLj41mrt1GjRtSqVSvV3X8dM2rUKOrfvz9ZW1tTSEgI7dmzRyOFTE1NJTc3t1q3u2zZMmIYRvFZtmyZ0msZGRm1bkNT3NzcKCUlRflFLbtOvUB6GAJCQ0PRv39/VuusLR999JHiO/J4PDAMAxMTE4SEhGDPnj2QSqVK77Ozs8Py5ct1avvixYsgIkRGRla7FhwcXO2oUx/PAgBOnz4NIlJ2tKp8OE5JSSEPDw+dtT8/P5/Wrl2reONmz55N2dnZOterCe7u7nTnzp06aasmuFyu4ueKigoCQMXFxbR792765JNPyNramkJDQ2nfvn1UUlKiKPv8+XOytrbWqW1fX1+Kjo6m1atXV+nxrl+/TkOHDiVbW1ud6tcUKysrIiLKycmpdq2a83tpaSk9e/aMFWfnr7/+mlavXk1ZWVlUUlJCzs7OlJOTQ6tWrdK57ppo0qQJPXz4UO/taIKZmZnSv1dUVBARKRRyx44dJBAIqH///tSvXz+qqKhgxce3W7duRER09OhRGjNmDBER/fHHHzRo0CCd69YUCwsLIiKSSCTVrlVTwsLCQiIiEgqFOjfcqFEjioyMrPK2rV69uk6U0NzcnCQSCe3cuZOIXvbKMplMcR0A5eXlVbmnoqKi2j+ptLSUpFJplb9JpdJqEQwkEolCqeTk5eURAKVv/+uUl5cT0UuF3Lp1K23dupWIiJVFiZeXF0VGRlJERAQNHjyYiIjS0tJYdQetCXlYEfn3fJVqSlhUVERExMqiRL7aysjIUChDXSHvfeT/dDMzs2rxVUQiUZWhkoiqOXRzuVwSiURV/mZkZFStd7O1ta32PxMKhcTn8+n48eOUnp5eo8wMwyiUzsvLi65cuaI2XIc2REZG0urVq+nw4cNkZmZGI0aMYKVeTZHrlampabVr1ZRQ/qAqKytZaXzt2rUUHx9P0dHRNH36dFbq1ISysjIievnm8XiGDbnz8OFDSkpKqtZTyuHz+VReXk5eXl702WefUUhICNnY2JCpqaliZNIVeW8YGhpKwcHBtH//flbq1RT5CKNselHt6ciH4YKCAp0b3r59O0VERFB6enqddv1EL7+0iYmJwRWQiJSeOskVr3HjxvTpp59SeHh4tcWgg4MDPX78mDU5RowYQatXr6bg4GCl1/Pz86v8LJ/HsYH8ezRu3LjaNaVKaGpqSs+ePdO54dDQUCKiOldAIqLs7Gxq1KhRnberDB6PRxUVFcTlcgkACQQCGjRoEI0cOVJtmBO2V/i+vr4UHBxMfn5+1a69bqwsFouJ6OXcmQ3u3LlDTk5OStcaSr+9m5sbpaam6tyw/I3LyMioUt+r2zSvv31s8c8//7CyzcQG8h6lW7du9Ntvv1FOTg5t2LCBunbtqnbh4enpSbdv32ZNjvz8fHJ0dFS6AQ5A6Yctbt++rfJ5KP0PsPXl5QuTtWvXklgspqioKIqMjFTshTEMo3jjiF6+fWy5DyQnJ9cbJZw/fz49efKEjhw5QmFhYRpHx+rSpQtdvXpV6bZGbYiLi6vTbZlXSUhIUNoDE5HyrfGYmBjY2trWS79WTSguLoaJiQm2b99uaFF0Ijc3F1wuF4cOHap1Ha8aJURFRbEoneY8fPgQRITz588ru6z8xKRr166UnZ1NycnJensz9MnFixeppKSEAgICDC2KTojFYvLz89NpJSufj8fGxurVQEEd+/fvJ3t7e+rQoYPyAqq018XFpcEaU06ZMgUdO3Y0tBis0FCNi1/F19cXU6dOVXVZtWX1/Pnz4enpqR+p9Eh5eTns7OywcuVKQ4vCCgUFBbCwsGiwU4tbt26BYRjcuHFDVRHVSpieng4ej4dz587pRzo9sW/fPgiFQjx//tzQorDGzJkz0bZt2wY5Rx82bBj69Omjroh6b7vhw4ejd+/e7EqlZ3x8fDB9+nRDi8Eq2dnZMDMzw/79+w0tilbcuXMHfD4fiYmJ6oqpV8Lbt2+Dz+fj8uXL7EqnJ+Lj4yEUCg0WMFKfLFy4EK6urg3KZaFnz54YOnRoTcVqjkUzZcoUeHt713vf4+LiYri4uGDx4sWGFkUvlJSUwMPDA1999ZWhRdGIzZs3QyQSaRK8qWYlzM/Ph4ODQ70NhiRn0qRJcHd3f+OiL8hJSUlB//79QUTYunWrocVRS0pKCszNzfHzzz9rUlyzqFy///47eDweEhISdJNOT+zZswc8Hg/NmjXD1atXDS0Oa9y/fx9LlixBq1atFBvORIRGjRrhn3/+MbR4SsnLy0Pr1q0xYMAATRdSmscn/OKLL2BjY1PvvvylS5cgEokQFRUFf39/mJiYNLioEa/y+PFj/Pjjj2jfvj0YhgGfzwfDMAoF3LNnD4KCguDi4lLvAshLpVIEBATA1dVVlaO7MjRXwtLSUnTv3h3NmzdHRkZG7aRkmTt37lQJnVtRUYGvvvoKDMNgzJgxDWYS/+zZM6xcuRIffvghGIZROEO92vvx+XxFKI2ioiJ06NABnp6e9eZZSCQSBAUFoUmTJtrmUdEuXLD8yzs7O+P27dvaSckyly9fhp2dHQIDA6vNA/fu3QuxWAxvb2/cu3fPQBKqp6KiAuvXr0dgYCC4XC64XC44HE4VxXv1IxQK8fjxY8X92dnZaN++PZycnHDz5k0DfpOXwRJ8fHzg6OiIW7duaXu79oHT8/Pz4e/vD2traxw9elTb21khLi4OQqEQ/fv3V+kumZqaCi8vL1hbW9fLmIUxMTEgomo9nrIPh8NRmpqhoKAAgYGBEAqFBgvjnJCQAEdHR3h4eODBgwe1qaJ2KSRKSkowbNgwcDgcREVF1VkeDalUigkTJoCIMGnSJFRUVKgtX1RUhBEjRoDD4WD27Nn1apuptLQUvr6+4PP5ahWQz+ejY8eOKif5ZWVlGDFiBBiGwfDhw+ssZLJUKsWsWbPA4/EwcOBAbeaAr6NbRqc1a9bAxMQELVu2xKlTp3Spqkb279+PZs2aQSQSYcCAAZBIJBrfu3r1ahgbG6N79+56yzpVGzIzM2FjYwMul6tWCdVNfXbu3Kko6+TkBLFYjOXLl+vN4KGyshI7d+6Es7MzxGIxG2f0uue2u3v3Lvr06QMiQs+ePVk/az5y5Ag6d+4MIsLQoUORlJQEKysrTJw4Uat6/vzzTzg7O6Np06Y1HSPVKefPn1ephFwuF/PmzVN6X3l5OaZPn64Yznv16gWJRIIZM2ZAIBDAwcEB0dHRrJ2hS6VSbNiwAZ6enuBwOAgPD2cri4DuSijn2LFjCmVp3749fvrpp1oLmZGRgSVLlqBNmzYgIvTo0QNnz55VXN+wYQM4HA4uXryoVb05OTn46KOPYGxsXC+sbCorKzF37txq2zByBVQV+jg7OxudO3dWKC+fz8eyZcsU1588eYIvvvgCQqEQxsbGGDBgALZu3ar1cWZeXh7279+PUaNGwcLCAnw+H8OGDUNycrLO3/0Vao7ery0XL16kDRs2UFxcHOXn51OrVq2oc+fO1KpVK3JzcyM7OzsyNzcnCwsLysvLI4lEUiXf8ZkzZyg1NZVsbW0pJCSEPv30U2rXrl2VNgBQjx49KDMzk/7+++9q/sTqkMlktGDBApo/fz4NHTqU1qxZo9QXVt/k5eVReHg4HTt2jH744QdKTEykrVu3KtxCGYahc+fO0QcffFDlvsTERPr444/pxYsXVRzJL1++TD4+PlXKSqVS2rt3L23ZsoXOnDlDUqmUXF1dydPTk9zd3RXPQiwWU0FBARUWFlJOTg6lpqZSSkoKJScnE8Mw5OvrSyEhIRQaGqpzWBIlsJfv+HVKS0tx6tQpfPvtt+jatStsbGzUTsAbN26Mbt26YcGCBTh37lyN4X7v3r0LU1NTzJ8/v1byHT58GNbW1mjTpo3WOZl1JSkpCS4uLnB0dFSYvBcVFaF169aKrRplAYxWrlwJPp9fbfgWCAQ1/r9KS0tx5swZLFy4EGFhYfDx8YGTkxMsLS1BRBCJRHBwcICXlxcGDRqEqKgoHDhwQKu5dy1hbzjWhBcvXuD27dv4888/cfz4cfz1119ISUlBfn5+rer73//+B2Nj41oPD+np6fD29oZIJKqzVGnbtm2DmZkZunTpUm14vH//vkKxXv2fFBcXIzw8XOl2DsMw8PPzqxPZ9UTdKiHbVFRUoH379vjwww9rbfBZUlKCzz//HAzD4Msvv9RbwPXy8nJMnToVRITJkyer3Na6d+8eUlJSqvzeunVr8Hg8pSOIkZER5s6dqxeZ64iGrYQAcPXqVfB4PJ0XGhs3boSpqSn8/PxYt0d8+vQp/Pz8YGpqii1btmh83+HDhxULAnVTmT/++INVeeuYhq+EAPD1119DJBJpe2ZZjWvXrsHV1RUODg6sWQxdvHgRjo6OaNGiBa5du6bxfX/88YdGJypcLhdFRUWsyGog3gwllEqlcHFxQd++fXWuKy8vD/369QOPx8P333+vk1/HqlWrYGxsjF69emmadlVBZmYmmjVrpvY8mYjQtm3bWstXT3gzlBAATp48CYZhEBcXp3NdMpkMS5YsAY/Hw6BBg1BQUKDV/cXFxRg1ahQYhsGcOXNqfVwok8mwatUqmJmZKR2SjYyMMG3atFrVXY94c5QQAEaPHg17e3utex1VnDx5EnZ2dvDw8NDYUiU9PR3t27eHWCxmLaHPw4cP0aRJE4Uxw6srYzXpuhoKb5YSvnjxAvb29hg1ahRrdT569AidOnWCUCis0az++PHjaNSoEet7jytXrgSPx8P//vc/NGrUSNErMgxT77OvasCbpYTASzMvhmFYTQRUVlaGKVOmgIgwYcKEatsr8uGby+ViyJAhrG7wPnr0CBYWFgoHp9zcXIwaNUpvUfYNwJunhADw8ccfw8XFRaWtYW2R2zH6+voqTOsLCgrwySefgMfjVTm/ZQtV3+X8+fP1yhBDB95MJXz06BFEIpFe3CNv374NT09P2NjYYP369fD09IStrS1Onz7NelvyXv3EiROs112PeDOVEPh3HnXlyhXW65ZIJOjUqZPChk/X/UllvHjxAo0bN2Z1fltPeXOVUCaToVOnTnjvvfdqtMDWhoqKCsycORMMw+DDDz8En89Hnz59dLEsVgrbK/16zJurhMDLodPY2Bj/+9//WKkvJycHQUFBEAgE+PXXXwEAFy5cgKOjI5o3b86az7N8z3PHjh2s1FfPebOVEHgZ4s7U1BRpaWk61fP333+jWbNmaNq0Kf78888q17Kzs+Hv7w+BQIB169bp1I48nAkbpz8NhDdfCUtLS9G6dWt069at1kdwGzZsgEAgQGBgoMp9uVeH6dGjR9fa55mtc/AGxJuvhMBLIwIul4sNGzZodV9paSnGjh0LhmEwY8YMjeaWv//+O8RiMd577z2tfZ7ZsghqYPw3lBB4GTDJyspKY5fIR48eoWPHjhAKhdi1a5dWbaWlpaFt27awsrLCgQMHNLrnVdvI+uSaWgf8d5RQIpGgadOmCAkJqbHsmTNnYG9vD3d391pbbUulUowcORIMw2D27Nk19qK6Wok3YP47SggABw8eBBGp7Z1+/PFH8Pl89O/fv9ZuB68SGxsLgUCA7t27Izs7W2mZu3fvwszMrNb+Mg2c/5YSAsDQoUPh5ORUzTyrqKgIQ4cOBZfLxcKFC1mND/3nn3/inXfeQdOmTXHp0qUq12QyGYKCgtC6des3NrZiDfz3lDA7OxvW1taYMGGC4m9paWl499139RpfJycnB7169YKxsTFiYmIUf5f7UF+4cAF5eXmYN2/eGxnuWA3/PSUEXvqTcDgcnD9/HgcPHoRYLEa7du1w//59vbZbWVmJ+fPng8PhICwsDA8ePIC1tTUmTpwImUyGfv36gYhqinavMxUVFUhLS8O5c+dw7NgxxMXF4fDhwzhz5gySk5PrukeuWyXMycnBzZs3kZiYiOPHj+Py5cu4deuWQY6munfvDltbW3A4HIwYMaJOYxkePXoU1tbWsLCwgIODAyQSCZYuXVrFn4QNC3E5OTk5iIuLw9ixY9GmTRsYGxurdRng8XhwcXFBeHg41q9fr+8YiPpTQqlUiuPHj2PmzJno1KkTrK2t1X5xW1tb+Pv7Y86cOTh9+rRe38bc3FwEBASAiAyWImP9+vUgIpiammLBggXVLKatrKx0iiNTVlaGuLg49OnTB3w+HwKBAF27dsXs2bOxdetW/PXXX3j06JGiA8jPz0dmZiaSkpKwa9cuLFq0CH369IFIJALDMOjUqRPWrFmjtauDBrCvhGfPnsXIkSNhYWEBDocDLy8vTJw4EatWrcKJEydw69Yt3L9/Hy9evMC9e/dw48YNHDt2DDExMYiMjETLli1BRLCyssL48eNZT1+RlJQEV1dXODo6YsKECTA2Nq5NYEedkEgkcHJyQkhICEaMGKE0qoI87ou2FBcXIyYmBs2aNQOfz0dwcDDi4uJqbVtZXl6O48ePY/jw4RAKhbCyssKcOXPYHL3YU8L4+Hj4+vqCiNCxY0esWLGi1qbnT548wffff68IiBQYGMiKvZ48+kHnzp2RmZmJiooKeHt744MPPqjTDeKJEyfC2toajx49qjFG4ZEjRzSu9/Dhw3BxcYGJiQnGjx+P9PR0VuV+8eIF5s2bB2trazRq1Ahr165lYxdBdyVMTU1FUFAQGIZB3759tY6UVRMnT55UDJ0DBw6s1Znqq9EPJk2aVMU8/9q1a+Dz+ZqmO9CZCxcugMPhYOPGjZg6dara2IQcDgeNGzeu0V1AIpEgPDwcRIRBgwbp/dw5Pz8fkydPBo/Hg5+fn64B3HVTwpiYGAgEArz77rt6z4Enf8tFIpFWUQyysrJqjH4wc+ZMmJub6/3hlZaWolWrVggKCkJcXJzaOfKri4Tx48errDMpKQlubm6ws7Or87DIV65cUViZa9Njv0btlFAqlWLw4MHgcrmYP3++3uK3vE5xcTGmTp0KhmEQGRlZY7uaRj8oLi6Gq6ur3rdG5s2bBzMzM9y9exfdunVTzP1qUkSGYarEZ5Rz9uxZiMVi+Pv7G2xvsaioCCNHjgSfz8dvv/1Wmyq0V8Lc3Fx06dIFtra2rHq0acPevXshEonQp08flSEwtI1+cOrUKb0akiYnJ8PY2BjR0dEAXvaKBw8exIgRI2Bubq5WIblcLlq0aFFlG+nEiRMwMTFBWFhYncUMV8eCBQvAMIzSAO81oJ0S5ufno0OHDmjWrJnBk+pcvXoV9vb2CAgIqLKd82r0g2+//VarBceYMWNgZ2fHeprayspKfPjhh2jfvr1SQwZNFJLH42HWrFkAXiYQMjc3x/jx4+tV+tnVq1eDy+Vqm0lAu2Q6AQEBcHFxYStWsc6kpqbC1tYWQ4YMQWVlZZXoB7VJy5qbm4vGjRvj008/ZVXOn3/+GTweTyPz/1cVUigUVlPIY8eOVUkgVN9YvHgxjIyMqp2Rq0FzJZw4cWK9TCuWlJQECwsLfP7556xEP9i1axcYhmEt3NrDhw8hEonw9ddfa31vaWkpDhw4gOHDh8PMzEyhiH5+fvXa2GHMmDFwdnbWdC9RMyXcs2cPjIyM6m0W+P3794PH46F///6sRD/o168fWrRowYrzfN++fVnJU1xaWorQ0FCIxeJ6MxKpoqSkBO3atUNoaKgmxWtWwtzcXNjb2+P//u//dJdOj0yfPh2urq6s5O94/PgxLCws8OWXX+pUz44dO1gLSXL//n2YmprWy+xUyrhz5w4EAoEm371mJRw/fjx8fX3rvcl5SUkJ3NzcWDMMXbVqFXg8Hv7+++9a3f/ixQvY2dlh9OjRrMjTr18/fPLJJ6zUVVfMmTMHHh4eNa3e1SvhrVu3dHoQdc3hw4dhampaJRFhbZHJZOjcuTPatWtXq33QUaNGoXHjxqycsSYmJoLH4+nstlrXSKVS2NraKny0VaBeCcPCwhAcHMyuZHrG19cXU6dOZaWulJQUCAQCLF26VKv7Tpw4AYZhsHPnTlbk6Nu3L8LCwlipq65ZsmQJXF1d1b3IqpXwwYMH4PF4uHDhgn6k0xO///47zMzMWMth991338HExETjXkgqlaJFixb4+OOPWWn/n3/+AcMw1RzuGwp5eXkwNTVVF8xTtRLOnTsXLVu21I9keqSiogL29vZVTOh1oaysDG3atEFAQIBG+3IzZsyAhYUFa1nZv/32W7Rq1YqVugzFkCFDFAnDlaBcCWUyGZo3b44lS5boTzI98sUXX8DX15e1+i5dugQul1tjiI8rV66Ax+Nh1apVrLXdokULLFq0iLX6DEF8fDyMjIxUBY1SroQ3b94EEek9o3h6ejoiIyNBRIiMjGQtDt/p06fBMAyePn3KSn0AMGXKFFhZWamss7y8HO+99x46derE2klGWloaiKhOFob6ehYAUFhYCCMjI1VDsnIlXLFiBezt7fV6LCTPICn/edu2bSCiWh23vU5JSQlMTEywbds2neuSI5FI4OzsjMGDByu9vnTpUhgbG6vNTawtsbGxsLS01Pv2mD6fhZzOnTtX8XB8BeVKOGTIEPTv3581AZSh7AuyGYdZzZeuNYcPH1b6cNLS0hS+IsqYOXMmiAjffPONVqPLZ599hu7du+sksybo+1kALw8T3mwW3rEAACAASURBVH//fWWXlCuhl5cXZs6cqXPDeXl5iI2NVXyhqKgotbFg5EMBG0RERKBbt26s1PUqYWFhaNKkicLhRyaToVu3bmjTpo3KTdng4OAqhgienp5YtGgRHjx4oLYtXV+k6OjoKpY4cjOy168pcwNg81kAL3t1kUik7JJyJTQ1NdU6gpUy5HOMrKwspKenq/1ieXl5rA4B0dHRcHJyYqWuV3n27BkaNWqEcePGAQDWrVsHLper1q1h6NCh1WwE5Ym233//fcTExCh9Oe3s7LB8+XKd5L148aLK/3twcLDSdtl+FsDLebpcF16juhJKJJIa47VoSlRUVJUvr66LP3HiBIKDg5GXl6dzu8BLB3eBQMBKXa/z22+/gcPhID4+HlZWVpg0aZLa8sqU8FU/Ei6XCy6Xi27dumHjxo2KGDg8Hk8rVwZVyHu9V3u8a9euqZwzs/0sgJfWTkSkzLMxg0evUVhYSEREIpHo9Utas2DBAiIiysjIoJ07d6ot++OPP9KsWbPIwsJC53aJiMzNzamkpIS2b99OXC6XlTrlCAQCatOmDQ0ePJjMzMzI29tb7ffLzMxUeU0mkyl+PnXqFJ08eZLGjBlD7du3p4qKCjI3N9dZ3m7duhER0dGjR2nMmDFERPTHH3/QoEGDlJZn+1kQkaIuiURS/eLraikfNuUZyXUlNjYWwcHBuHPnjsqecNu2bYiNjWWlPTkHDhzQyJGoLj5isbjW97IxIgH/To3y8vKQl5enclqkj2cBvEwYSUTKfGUyOK8rpZmZGRERlZSUaKzlqti+fTtFRERQTEwMubm5KS1z/fp1unXrluINZYuioiIiIgLA+qegoICaNGlC7733HhkZGdHNmzfVlu/Vq1eN8nK5XOJwOMTn86lv3760fft2IiIqLS1l5f8RGRlJRESHDx+mhIQEGjFiRLUy+noWRP8+D1NT0+oXX1fLkpISEBF+//13nbWfXuv5Xv89KyuryooNeDlXYWNV9ssvv6hajenMhAkT0KhRI2RmZsLHxwcdO3ZUu5enak7IMAx4PB44HA66du2KdevWVZmHmZiYYOPGjazJLe8NlRml6PNZAC99gohImdW78tWxra0tVqxYoXPD8q2J9PT0KsNxVlYWsrKyFNdf/7CxKpszZ45ezlzPnz8PDoejcG+8fv06+Hy+2v/X60oo36pp3749fvrpJ5Xummwf2clXyq8Pt/p+FsC/0yMllu/KlZCtjd5r165V2R+Ur5ZfPSJS9rlz547ObYeGhrK+4V5SUoKWLVtW20D+5ptvYG5urjJ6Vc+ePRXfzdXVFd999x3u3r1bY3u9evXCiBEj2BBdgXx+/ir6fhYAsGzZMlVbZsqVcNy4cfDz82OlcUPRqlUrfPvtt6zWOWfOHJiZmVWLyl9cXAw3NzeVEb4OHTqEiIgIXL9+Xav2pk2bpuqUoVaoW5Dom9GjRyMoKEjZJeVKuGPHDggEgjqN2ccmWVlZYBgGp06dYq3OW7duwdjYGN9//73S62fOnAHDMKyeV//+++/g8/mshWOLjY01WLJGNzc3fPfdd8ouKVfC7OxscDicBptdcvv27TA1NWXtJaqsrMQHH3wAHx8ftVH4IyIiYGtry5rzfG5uLrhcLg4dOlTrOqKiohRDa1RUFCtyacvDhw/VbfupNmoNDAzE559/rj/J9MiAAQMwZMgQ1upbsWIF+Hy+2ng2wMvhzsHBgdV5XEBAgE5DqPzsXh97f5ry888/w97eXtULrFoJN27cCEtLS1ZcKOuS58+fw8jIiDXXyIcPH8Lc3Fxjg47du3eDYRgcO3aMlfbXr18PKyurBvccXqUGvx/VSqihp1S9QwPHGq3o06cP3NzctBraBwwYgObNm6sM1qQNBQUFsLCwwPbt23WuyxDcunULDMPgxo0bqoqo97ZbvHgxXF1dWc0XrE/YfnG2b98OhmG0jhL7+PFjiMViTJ8+nRU5Zs6cibZt29bL2DM1MWzYsJpC7qlXwvz8fNjb22PNmjXsSqYn5s+fr4mztUY8f/4ctra2GDNmTK3uX7NmDbhcLv766y+dZcnOzoaZmRmrplV1wZ07d8Dn85GYmKiuWM0RGDZt2gRra2uNExMairS0NJiYmLCWDOfTTz+Fg4NDrTO6y2QydOnSBW3btmVlarBw4UJWYtrUJT179sTQoUNrKlazEspkMnTt2hU9evSot8NBaWkpfHx8WAuTcfz4cTAMg927d+tUjzweCxteiyUlJfDw8MBXX32lc111webNmyESiTQJ3qRZVK4HDx7A0tKy3iYAnDx5MmxsbODo6KizCVpRURFatGihzk9WKxYtWgQTExOdQuqVlJQgPj4e9vb2YBgGhw8fZkU2fZGSkgJzc3NNg9FrHp9w37594HA42kbh1DvyoIyHDx9GcHCwzpH4p0+fDgsLC1bi2QAvneffffdd+Pv7azWSyBUvPDxcESyTiGBvb18v40TKycvLQ+vWrTFgwABNv6924YJ/+OEH8Pl8vcV11pbY2NgqL4ZMJsO8efPA4XAwfPhwreML/v333+DxeKwvxC5fvgwul4tffvlFbbnS0tJqiie3uGEYRnEaExQUBBcXF9aiPLCFVCpFQEAAXF1dtZlLax84/dtvvwWXy62zvB+qkCcqVHaWe/DgQVhaWqJdu3bVjA1UUV5ejnbt2qFz5856mftOnToVlpaW1cy2Xo3Gqiw88KsfuY1nUVEROnToAE9PT33nndMYiUSCoKAgNGnSRNtUHLVLIfHDDz+AYRhERETU+WotPz8fgwYNAp/PV2vwmZaWBi8vL1hbW2uUY2PJkiUQCARISUlhU1wFhYWFaNasGQYNGgTgpZHnsGHDalQ8+bXXne6zs7PRvn17ODk56T1SRk1kZWXBx8cHjo6OtUnRVvtkOvv27YNYLIaXl1edxS88c+YMXF1dYW9vr1H006KiIoSFhdWYSFu+vaPCyoM1jhw5AiLCvn370LRpU418TBiGgVgsVrpFVlBQgMDAQAiFQoPN1RMSEuDo6AgPD48a/ahVoFtGp3v37qFz587gcDgYP3683vYS09PTER4eDoZh0KtXL60Tx/z000/g8/n4+OOPFe6UcmQyGQICAvDuu+/WST6Q8PBwNGnSBElJSTA3N6+S3VOVEm7atEllfWVlZZg2bRoYhsHw4cPrbD9XKpVi1qxZ4PF4GDhwYK33U8FGbjuZTIYNGzagcePGMDU1xZQpU1iLKHr79m189tlnMDIygrOzs7oYdzWSkJAAe3t7uLu7Vxkyfv31V3C53Jp29Vnj2bNnsLGxwdixY3HgwIEqOY6VDcM9evTQqN6DBw/CyckJYrEYy5cv15vBQ2VlJXbu3AlnZ2eIxWKsXLlS1yrZy/IplUqxfPlyODk5KfLjrlixAsnJyRrXIZPJcP36dSxbtgzt27dXmMP/8ssvrKRMePToET744AMIhULs3LkTT58+haWlJaZMmaJz3dqwefNmRaqwadOmqewNTUxMtFp4SCQSzJgxAwKBAA4ODoiOjmbNtlEqlWLDhg3w9PQEh8NBeHg4W1kEMhgAqO6DV3tkMhmdPHmSNm3aREeOHKHs7GyysbGhli1bkru7O9nZ2ZFQKCSxWEy5ubkkkUjo6dOnlJKSQsnJyZSbm0sODg4UHBxMQ4cOpc6dOxPDMKzJV15eTlOmTKFVq1aRm5sblZaW0o0bN0goFLLWhib07NmTUlJSiMPhUHZ2NpWWllJ5ebniOofDoZiYGBo7dqzWdWdmZlJ0dDTFxsZSeXk59e7dmwYOHEj+/v5kb2+vcT35+fmUkJBA+/bto927d5NUKqWQkBCaNWsWeXp6ai2XCh6yroSvAoBu3rxJCQkJlJycTKmpqfT06VMqLCyk/Px8EovFZG5uTo0bNyZ3d3dq2bIl+fn5kYeHh75EUjB16lT68ccfqW3btnTs2DGysbHRe5uvsmzZMpo+fTo1a9aMDh06RAEBAfTs2TOqrKwkPp9P3t7edP78eZ1eQKlUSnv37qUtW7bQmTNnSCqVkqurK3l6eio6BHNzcxKLxVRQUECFhYWUk5NDqampik6BYRjy9fWlkJAQCg0NJWtraxb/C0SkbyWsrxQUFFCrVq3o3XffpeTkZJLJZLRr1y7y8fHRe9slJSU0adIk+uWXXygwMJASEhLo77//psLCQurcuTNVVFQQn8+nW7dukaurK2vtlpWV0aVLl+jcuXPVOoTc3FwSiUQkFArJxsaG3NzcyN3dnXx9fcnPz0/fo8RD9gLQNSDGjh2LRo0a4dmzZ8jJyUGPHj0gEAhqPNHQlXv37sHb2xtisRj79u1DRUUFOnToAF9fX1RUVCAmJgZE1ODDA2sJewuThsK5c+eqnYFXVFRg1qxZig14fawsDx48CGtra3h5eVWJQpCUlAQ+n68IAXfjxo0GY0TMEv8tJSwpKYGnp6fKbQ95HuX333+ftSzwlZWVmD17NjgcDkaMGKHU5D8qKgpCoVBpsMr/AP8tJZw9ezaEQiHu37+vskxKSgo8PT1hY2Ojs99yTk4OunfvDmNjY7XJqEtKSuDu7o5evXrp1F4D5b+jhDdv3oSRkRF++OGHGstKJBIMHDgQPB6vWpAgTUlMTETTpk3h7OysUSKchIQEMAzDSlDMBsZ/QwkrKyvRsWNHdOjQQeP5lkwmw9KlS8HlchESEqJVCtuVK1fC2NgYH330kVaZpSIjI2Fra8taNqoGwn9DCZcvXw4+n691LBgA+OOPP2BjY4NWrVrVmMy7qKgIw4YNA4fDwdy5c7VO/ZCXlwdHR0cMHz5cazkbMG++EmZkZEAoFOKbb76pdR3p6enw9vaGhYWFSo+31NRUtGnTBtbW1jqZ3+/duxdExJrDVgPgzVfC3r17w93dXWe7x+LiYnz22WdgGAazZ8+u0svt2bMHIpEI3t7erKxwP/nkE7zzzjsoLCzUua4GwJuthFu3bgXDMEhISGCtzjVr1sDY2Bg9e/ZEVlYWvvzySzAMg88//5y1/cUnT57A0tIS06ZNY6W+es6bq4Q5OTmwtbVFREQE63VfunQJjRs3hkAggEAgYDWkr5zY2FhwudwGm2JWC6oHTn9TmDZtGvF4PFq6dCnrdZeVlREAhXEBh8P+v3H06NHUuXNnGj16dBXrmjcSQ78G+uDYsWNgGEYnI1hlyGQyfP/99+DxeOjXrx9ycnLwxRdfgIgwadIk1i2zU1NTIRAIsHjxYlbrrWe8ecNxYWEhmjdvjgEDBrBar9zBisfjYcmSJVX8VbZv3w6hUIhOnTpp7XpQE4sXL4ZAIKiyPZSfn48xY8Zgzpw5rLZlIN48JZw2bRrEYjFrzuvAy9MWDw8P2NnZqXSwunHjBlxdXeHg4IBz586x1nZ5eTm8vLzQtWtXyGQy7N+/H3Z2dgrr6/oamkUL3iwl/Ouvv8DlclmNSrplyxZFL1eTs3lubq4iCkRMTAxrMvz555/gcrnw8fEBEVVxB2Azv7KBeHOUsLy8HG3btlX0GLpSVlaGCRMmgIgwZcoUjed7ukaBUFbfunXrYGJiUs03mcvlYu3atTrVXw94c5RQPndiI+/Go0eP4OvrC6FQiLi4uFrVcejQIa2jQLxOWloaunTpAoZhlHrl8Xg8hIeH16ruesSboYT//PMPBAIBKxbJ8rNiT09PnYe6u3fvKqJAaHuUd+HCBUVv97ryvfpp0qSJTjLWAxq+EspkMvj7+8PLy0unLRKZTIaFCxeCy+ViyJAhWlnNqOPVKBDfffedxlOFlJQURW+nTgmJqN4FRtKShq+Ea9eu1flkITc3F3369AGfz8dPP/3EonT/oi4KhCqeP3+OHj16qI3SwDBMgw2q/v9p2EooP2NVk56gRq5evYrmzZvD0dERFy5cYFG66pw9e1ZpFIhX+eGHHxTJG4GXtpBz5swBwzBKldHIyIiVPIQGhH3nd2VkZmZWcTMsKioiiURCIpGIzMzMyMHBgdzc3Khly5Zka2urcb2DBw+mv/76i27cuKHI06wN69evp3HjxlHHjh1px44ddeJ7/OTJExo0aBAlJSXRunXrqmRf3717Nw0cOJCIiK5du0ZeXl6KawcPHqQhQ4ZUc5InIvL09KTk5GSNZaisrKQHDx7Q06dPSSqVUl5eHpmbm5OpqSnZ2NhQixYtyMjISMdvqjH6cfksKSnBnj17MGbMGLi4uCjeWmtra7Rq1Qrvv/8+unXrBh8fH7Rs2RKWlpaKMi1btsS4ceNw6NAhtVbQ+/btq7XdXXFxMUaPHg2GYTBz5sw6924rKyvDuHHjwDAMpk+fjoqKCiQnJ8PExESRA7ldu3bV5EpLS0PLli2rzRMZhlEbkCgnJwdxcXEYO3Ys2rRpA2NjY7VzTB6PBxcXF4SHh2P9+vX6joHI7nB87949TJw4EVZWVuByuQgMDMSiRYtw9uzZGk3Ws7KycPLkScydO1cR6cve3h5ffvlltdMPuQVybbYn7t27h/feew9isZiVxOK6sHHjRpiYmKBLly5o1qxZFeXicDj48ccfq90jX+i8vmXzegarsrIyxMXFKea6AoEAXbt2xezZs7F161b89ddfePToEV68eAHg5VFgZmYmkpKSsGvXLixatAh9+vSBSCRSxBZas2YNa8keX4EdJXz48CHCw8PB4/HQvHlzfP/99zoHy0lPT8eCBQvg6OgIY2NjjBs3Ds+ePQPw0hfDxsZG8bumHDhwAFZWVmjbti1rkcN05e+//4aNjY3SVbCJiYnKmH8xMTHgcrngcrng8XiKtGfFxcWIiYlBs2bNwOfzERwcjLi4uFpvmpeXl+P48eOKSLJWVlaYM2eOQnlZQDclrKioQHR0NIRCIVxcXLB582bW0nnJKSkpwZo1a+Dg4ABra2ssXLgQHA5HK6+0iooKzJ49GwzDYOTIkTqfYrDJokWLVK5+awoNd/78edja2oKIIBKJcPjwYbi4uMDExATjx49n3Y/5xYsXmDdvHqytrdGoUSOsXbuWjdOp2ithZmYm/P39IRAIMHfuXL2HDZZIJJg6dSp4PB4CAgI0DsqYnZ2N7t27QyAQGDTTpTKOHj1aY5BMIlKbQzkzM7NK2UGDBrHmuK+K/Px8TJ48GTweD35+frruU9ZOCa9du4bGjRvD1dUVV69e1UUArZGHp23RokWNaRQuXbqEpk2b4p133ql3FsoPHjyoZoyg7MMwDKysrFTGGUxKSoKrqytsbGxYy2yqKVeuXFEECtAkLrgKtFfChIQEiMVi9O7dW+NNV7bJzs5Gp06dYGdnpzJedkxMDIyNjdGrV6966cd74sSJKoYI6hSRz+crzaF89uxZiMVi+Pv7s27HqClFRUUYOXIk+Hx+lf1NLdBOCRMTE2FhYYHw8PA6ie+sDqlUir59+1ZLLCNfPXI4HMyfP19r39+6JDc3F+vWrUNgYKBigaGqZ2QYpoot44kTJ2BiYoKwsDCDPwsAWLBgARiGURvuRAWaK+G9e/dgY2ODsLCwemNIWVpaih49eqB58+bIysrCnTt30Lp1a1hbWzc4v93s7GysXLkSH374oWKv8NVtGC6XC2dnZxQXF+PSpUswNzfH+PHj682zAIDVq1eDy+Vqm0lAMyUsLS2Ft7c3AgICWIkdzSZFRUV4//338f777ysiajX06FaPHj3CDz/8gPfeew8Mw1SxIxw3bly96wxeRZ7m7dKlS5reopkSTps2DU2bNq2Xcyvg3wSQgYGBeotabyju3buHxYsXo2XLlgpF7NChQ73rDF5lzJgxcHZ21nQvsWYlvHr1KoyNjXH27FndpdMjv//+O0xNTWub0KVBMGXKFJibm9ebVGKqKCkpQbt27RAaGqpJcfVKKJPJ0LFjR4wdO5Yd6fTMJ598wlqK2PrG/fv3YWpqWufbMLVFnutZg8xb6pVw9+7dMDc3Z/OIRq/cv38ffD6/3vfataFfv36sJRWvK+bMmQMPD4+aVu/qldDb2xszZsxgVzI98+mnn6Jnz56GFoNVEhMTwePx6s15t6ZIpVLY2tri119/VVdMtRKePn0aPB6PVf/duuDmzZsgIty4ccPQorBG3759ERYWZmgxasWSJUvg6uqqzqZAtRKOHDmywfYo3t7e+PLLLw0tBiv8888/YBim3h07akpeXh5MTU3VhWRRHhCppKSEdu/eTWFhYZrbx9YjwsLCaNu2bYQ3IE/Qpk2bqGXLluTt7W1oUWqFhYUF9e3blzZu3KiyjFIlvHTpEhUWFlJQUJDehCMiys7OptmzZxPDMMQwDG3fvp2VeoOCgujRo0d069YtVuozJFu2bGmwnYGcsLAwOnToEOXl5SkvoKx/jIqKQps2bfTWRQMvLakvXryo+H3btm0golpHy38de3t7RYKahkpaWhqIqE6SmmdlZSEqKkoj8zFtKSwshJGRkaohWfmcsFevXkqtNtjkVQWUI/8HsEGfPn3qTXSCGzduIDY2Vuu0r7GxsbC0tNS7EYa+OwQA6Ny5syqvQOVzwpSUFHJ3d9e5G87Pz6e1a9cqhtvZs2dTdnY2ERH5+vpWK0tEFBUVpXO7RETu7u6UmprKSl268v3331NERATZ2dlRcHAw7d27l0pLS2u8LzExkXx8fPQShPNV7t27V+V5DBkyhIiIpk+fzlobHTp0oMuXLyu9Vu3byWQySk9PZyXD5Ndff00RERGUlZVF6enp9N1339GcOXOqlcvIyKDo6GgiIgoPD9e5XSKid955h+7evctKXboiEAiIx+NRRUUFHTlyhD755BOytbWlcePG0cWLF1Xel5qaSm5ubrVud9myZYoOgGEYWrZsmdJrDg4OVe5ju0MgInJzc6OUlBTlF1/vG/Py8kBEuljKKoiKikJkZKTid1Iy3Kanp1exm2NrCNiyZQu4XC4rdenKsGHDlDoyya1jmjZtivnz5+Pu3btV7rOzs9N5Xnvx4kUQUZXnICc4OBhZWVlV/paenq6YG7IRXErO6dOnQUTV2gOQwXtdKQsKCoiIWMlxu2DBAiJ62dPt3LlTaZmmTZsSALp+/Trt2rWLpk+fTiKRiMaMGaNT2xYWFlRZWUmbN28mY2NjEggEZGJiUqWMmZlZNSdvkUhEXC63yt8sLS2r/M7lckkkEmksi0wmU7pdJHdiz8jIoAULFtCcOXOoY8eONGLECAoJCaHnz5/rnOTa19eXoqOjafr06TRz5kxq2rQpERFdv36dhg4dWiXYQEZGBjk7Oyt+j4+Pp2nTpunUvhwrKysiIsrJyake4KCaWmZkgIhYizYaGxuL4OBg3Llzp8aFhyZlNCU+Pr5GByI2P0ZGRrC0tKzycXBwQPPmzeHg4KBxPa9bVqtK3qMN165dAxFVcfSKjo5WaXd57do1RW/IlnOYfMRTYmdYvSeUv+GFhYXaq/trbN++nSIiIig9PV3xBqpDl/nP6xQUFBCPx1P0NkVFRVRWVlatTGVlZZW/5ebmVvm9srJSMTrIKSsro6Kioip/Ky4uppKSkip/KywspPLyctq7dy89efJEI7k5HA7JZLIqv+uKl5cXRUZGUkREBA0ePJiIiNLS0lQ+Ey8vLzIxMaHvvvuOIiIidB6ViEgx4ijLRKBUCblcbrV/fG0IDQ0lItJIAYn+nRBv27ZN57Zzc3NJLBYrfjczM6sWr+b1YVZfJCYmqr3O5/OpoqKCTExMqF+/fjR48GAKCgoiMzMzjVbRmhAZGUmrV6+mw4cPk5mZGY0YMUJteTY7BCJSvLSmpqbVrlVTQoZhqFmzZpSWlqZzw8HBwRQfH08ZGRlVeons7GwaPXo0+fn50aBBg6hp06aUn59P0dHRFBUVpdgi0IV79+6Ri4uLzvWwwes9MNG/iicQCOjjjz+mIUOGUI8ePUggECjKmJiYsDIiEf3bG4aGhlJwcDDt379fbXk2OwQiIolEQkRE5ubm1S8qG7979erFSqZJ+VwkKipKsSMfGRmJ9PR07N+/v9qqWNkGdm3p3bt3vcmW+cEHH4CIFM5LJiYmCAkJwZ49e9QGDWjRogUr0WflyFfKr8/zgoODq8wR8/LyEBUVhaioKNbaPnDgAIhIWfBR5Scm3377LVq1asWaAHWNTCaDvb09VqxYYWhRALx8yESEwYMHY8+ePRqHIdHHyZV8kfgq+u4QAGDZsmVwcnJSdqn6woSIyN/fnxYsWEDZ2dlaxQusLyQnJ9PTp0/J39/f0KIQEdHOnTupvLxc620vT09POnv2LGty5Ofnk6OjY7X5XnBwsN4tjm7fvk0eHh5KryldenXs2JHMzc3p2LFjehVMXxw9epScnJyoVatWhhaFiIiMjY1rte/apUsXunr1qmI+pStxcXFVgnLWJQkJCeTn56f0mlIlNDY2poEDB9KWLVv0Kpi+2LJlCw0dOtTQYuhMly5dSCaT0blz52pdx6umchkZGRQQEMCihJrx6NEjSk1NVT0yqRrDExISwOVyG1xk+Bs3boBhGNy8edPQorBCQECA0iM3TYmNjWV107k2/Pzzz7C3t1cVEVe9o5OPjw+mT5+uH8n0xIgRI9C7d29Di8Ea69evh5WVVYN26vf19VUX3F69Eu7btw9CoVDriKiG4u7du+DxeHqPwl+XFBQUwMLCosGmibh16xYYhlHneKY+ej8A8vPzI09PT1qzZo2eZgzs0b9/f+Lz+RQXF2doUVhl1qxZdPjwYbpy5Yoi0XdDITw8nPLy8ig+Pl5VkZqj9yclJcHIyAhnzpxh8f1gn927d0MoFDb4YEjKyM7OhpmZGSvGDHXJnTt3wOfzkZiYqK6YZgGRvvrqKzg4OCA7O5sd6Vjm/v37EIvFrJqj1zcWLlwIV1dXvYdlZpOePXti6NChNRXTPDRchw4d6mVouPz8fHh7e6NXr171MlQaW5SUlMDDwwNfffWVoUXRiM2bN0MkEmmSxUHzIJn3799XxMWr6+QzqigtLUVAQABcwwwKaAAAA4RJREFUXFwazOJJF06dOgUej6d1xtC6JiUlBebm5vj55581Ka5duOBr167BwsICQ4cONXiPWFRUhD59+lQLF/ymM3v27Hr9nfPy8tC6dWsMGDBA05FJ+8Dp586dg6WlJbp37468vDztpWSBzMxMdOjQAfb29nWePcDQVFRUICgoCC4uLvXuIEEqlSIgIACurq4ap/hAbVNI3LhxA02aNDFIaoaTJ08qMmXWNqN6Q6eoqAgdOnSAp6dnvQmYKZFIEBQUhCZNmmibR6X2yXTkSWqMjIzwzTff6D1LUm5uLiZOnAgul4tBgwYZLH1FfSE7Oxvt27eHk5OTwY8os7Ky4OPjA0dHR5UpdNWgW1qxyspK/Pjjj7CwsICzszPWrVvHejoDea42e3t72NjYYP369azW35ApKChAYGAghEKhthHzWUOe3MjDw6O2oZrZSbD45MkTjB49GkZGRmjWrBmWLFmic1zDu3fvYv78+WjcuDFMTEwwadKkBhMxti4pKyvDtGnTwDAMhg8frsyvVy9IpVLMmjULPB4PAwcO1GYO+DrspppNT0/H5MmTYWtrCw6Hgy5dumDevHk4depUjRvdT548wfHjxxEVFQVfX18wDAMHBwfMnDkTT58+ZVPMN5KDBw/CyckJYrEYy5cv15vBQ2VlJXbu3AlnZ2eIxWKsXLlS1yrZVUI5ZWVliI+Px7hx4+Dh4aEwG7e0tISHhwe8vb3RrVs3tG/fHu7u7hCJRIqsRW3atMHkyZPxxx9/1Jv9yIaCRCLBjBkzIBAI4ODggOjoaK2DMKlCKpViw4YN8PT0BIfDQXh4uM7phP8/6g0Y2CI7O5uSk5MpNTWVnj59SoWFhZSfn09isZiEQiE1btyYPDw8yNPTU+eIA28hyszMpOjoaIqNjaXy8nLq3bs3DRw4kPz9/cne3l7jevLz8ykhIYH27dtHu3fvJqlUSiEhITRr1izy9PRkS9yHdaKEbzEMUqmU9u7dS1u2bKEzZ86QVColV1dX8vT0JHd3d7KzsyNzc3MSi8VUUFBAhYWFlJOTQ6mpqZSSkkLJycnEMAz5+vpSSEgIhYaG6qOTeKuE/xXKysro0qVLdO7cuWqjUm5uLolEIhIKhWRjY0Nubm7k7u5Ovr6+5Ofnx0pcIjW8VcK3GJyH+o2++Ja3aMBbJXyLwXmrhG8xODwi+sLQQrzlP43k/wEzzbezzTg4HgAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![a.png](attachment:a.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 实现softmax函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.34985881, 18.17414537, 54.59815003])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 逐步实现\n",
    "import numpy as np\n",
    "\n",
    "a = np.array([0.3, 2.9, 4.0])\n",
    "\n",
    "exp_a = np.exp(a) # 指数函数\n",
    "exp_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74.1221542101633"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_exp_a = np.sum(exp_a) # 指数函数的和\n",
    "sum_exp_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01821127, 0.24519181, 0.73659691])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = exp_a / sum_exp_a\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将softmax定义为函数\n",
    "def softmax(a):\n",
    "    exp_a = np.exp(a)\n",
    "    sum_exp_a = np.sum(exp_a)\n",
    "    y = exp_a / sum_exp_a\n",
    "    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$y_k = \\frac{exp(a_k)}{\\sum_{i=1}^{n} exp(a_i)} = \\frac{Cexp(a_k)}{C\\sum_{i=1}^{n} exp(a_i)}\n",
    "= \\frac{exp(a_k) + logC}{\\sum_{i=1}^{n} exp(a_i + logC)} = \\frac{exp(a_k +{C}')}{\\sum_{i=1}^{n} exp(a_i + {C}')}$$\n",
    "\n",
    "首先，在分子和分母上都乘上C这个任意的常数。然后，把这个C移动到指数函数(exp)中，记为logC。最后，把logC替换为另一个符号C'。\n",
    "\n",
    "\n",
    "这个式子说明，在进行softmax的指数函数的运算时，加上（或者减去）某个常数并不会改变运算的结果。这里的C'可以使用任何值，但是为了防止溢出，一般会使用输入信号中的最大值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: RuntimeWarning: overflow encountered in exp\n",
      "  after removing the cwd from sys.path.\n",
      "E:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: RuntimeWarning: invalid value encountered in true_divide\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([1010, 1000, 990])\n",
    "np.exp(a) / np.sum(np.exp(a)) # softmax函数的运算\n",
    "# 数值太大，无法被正确计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0, -10, -20])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = np.max(a) # 1010\n",
    "a - c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.99954600e-01, 4.53978686e-05, 2.06106005e-09])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(a-c) / np.sum(np.exp(a-c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如该例所示，通过减去输入信号中的最大值(上例中的c)，我们发现原本为 **nan(not a number，不确定)** 的地方，现在被正确计算了。综上，我们可以像下面这样实现softmax函数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(a):\n",
    "    c = np.max(a)\n",
    "    exp_a = np.exp(a-c) # 溢出对策\n",
    "    sum_exp_a = np.sum(exp_a)\n",
    "    y = exp_a / sum_exp_a\n",
    "    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.01821127 0.24519181 0.73659691]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([0.3, 2.9, 4.0])\n",
    "y = softmax(a)\n",
    "print(y)\n",
    "np.sum(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
